package com.dxf.bigdata.D05_spark_again

import org.apache.spark.broadcast.Broadcast
import org.apache.spark.rdd.RDD
import org.apache.spark.util.LongAccumulator
import org.apache.spark.{SparkConf, SparkContext}

import scala.collection.mutable

/**
 *
 */
object 广播变量 {

  def main(args: Array[String]): Unit = {

    val sparkConf = new SparkConf().setMaster("local[*]").setAppName("app")
    sparkConf.set("spark.port.maxRetries", "100")
    val sc = new SparkContext(sparkConf)

    val rdd1 = sc.makeRDD(List(("a",1),("b",2),("c",3),("d",4)), 2)
    val rdd2 = sc.makeRDD(List(("a",4),("b",5),("c",3),("d",6)), 2)

    // shuffle 性能不是很好
    val value: RDD[(String, (Int, Int))] = rdd1.join(rdd2)


    value.collect().foreach(println)

    println("===========================================================")

    val rdd3 = sc.makeRDD(List(("a",1),("b",2),("c",3),("d",4)), 2)
    val rdd4 = sc.makeRDD(List(("a",4),("b",5),("c",3),("d",6)), 2)
    val map4: mutable.Map[String, Int] = mutable.Map(("a", 4), ("b", 5), ("c", 3), ("d", 6)) // 放内存,但是每个Executor都要有一份map,冗余

    rdd3.map{
      case (w,c) =>{
        val i: Int = map4.getOrElse(w, 0)
        (w,(c,i))
      }
    }.collect().foreach(println)

    println("===========================================================")

    val rdd5 = sc.makeRDD(List(("a",1),("b",2),("c",3),("d",4)), 2)
    val rdd6 = sc.makeRDD(List(("a",4),("b",5),("c",3),("d",6)), 2)

    //广播变量 共享只读变量
    val broadcast: Broadcast[mutable.Map[String, Int]] = sc.broadcast(map4)


    rdd5.map{
      case (k,v)=>{
        val broadcastMap: mutable.Map[String, Int] = broadcast.value
        val bV: Int = broadcastMap.getOrElse(k, 0)
        (k,(v,bV))
      }
    }.collect().foreach(println)


    sc.stop()

  }

}
